import scrapy
import re
from ..items import TransactionPolicyItem


class ShggzySpider(scrapy.Spider):
    name = "shggzy"
    allowed_domains = ["shggzy.com"]
    # start_urls = ["https://www.shggzy.com/zcfgzhfg"]
    # start_urls = ["https://www.shggzy.com/zcfg"]
    start_urls = ["https://www.shggzy.com/xxgkgz"]

    def parse(self, response):
        node_list = response.xpath('//div[@class="gui-title-bottom"]/ul/li/@onclick').extract()
        for node in node_list:
            url_end = re.search("'.*'", node).group().replace("'", "")
            url_start = re.search("/[^/]+(?!.*/)", self.start_urls[0]).group()
            url = self.start_urls[0].replace(url_start, "") + url_end
            yield scrapy.Request(url=url, callback=self.parse_detail)
            # break
        for i in range(1, 9):
            if i == 1:
                url = self.start_urls[0]
                yield scrapy.Request(url=url, callback=self.parse)
            else:
                url = self.start_urls[0] + '_' + str(i) + ".jhtml"
                yield scrapy.Request(url=url, callback=self.parse)

    def parse_detail(self, response):
        title = response.xpath('//div[@class="content-box"]/h2/text()').extract_first()
        title_p = response.xpath('//div[@class="content-box"]/p/text()').extract_first()
        fabu_time = re.search("(\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2})", title_p).group(1)
        source = re.search("信息来源：(.*?)\s", title_p).group(1)
        url = response.request.url
        content_list = response.xpath('//div[@class="content"]//text()').extract()
        content = "\n".join(content_list)
        content = content.replace("EMSP", "")
        print(title)
        print("*" * 50)

        item = TransactionPolicyItem()
        item["title"] = title
        item["fabu_time"] = fabu_time
        item["url"] = url
        item["source"] = source
        item["content"] = content
        yield item

